{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6cbd0d0d-2d17-44b9-ae3b-76b413b760c3",
   "metadata": {},
   "source": [
    "## Create SparkContext in Apache Spark version 2.x and later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0508898d-8b48-4b16-bae9-87cead0489b1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "25/01/17 19:56:06 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "\n",
    "# Create a SparkSession\n",
    "spark = SparkSession.builder \\\n",
    "    .appName(\"MySparkApplication\") \\\n",
    "    .master(\"spark://spark-master:7077\") \\\n",
    "    .getOrCreate()\n",
    "\n",
    "# Get the SparkContext from the SparkSession\n",
    "sc = spark.sparkContext\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "82c7ca1e-dd73-4306-ab7f-01bc371eb94e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://docker-cluster:4041\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.2.1</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>spark://localhost:7077</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>MySparkApplication</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<SparkContext master=spark://localhost:7077 appName=MySparkApplication>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9a3361a7-cb9d-49a3-b80b-820e490711e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Shut down the current active SparkContext\n",
    "sc.stop() #or spark.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10336d77-bc36-4101-8055-dd7a2496d4da",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pyspark",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
